{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-10-12T13:59:55.648010Z",
     "iopub.status.busy": "2025-10-12T13:59:55.647717Z",
     "iopub.status.idle": "2025-10-12T13:59:55.676290Z",
     "shell.execute_reply": "2025-10-12T13:59:55.675404Z",
     "shell.execute_reply.started": "2025-10-12T13:59:55.647943Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "input_dir = './'\n",
    "working_dir = './'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-10-12T13:59:55.678761Z",
     "iopub.status.busy": "2025-10-12T13:59:55.678447Z",
     "iopub.status.idle": "2025-10-12T13:59:55.685783Z",
     "shell.execute_reply": "2025-10-12T13:59:55.684455Z",
     "shell.execute_reply.started": "2025-10-12T13:59:55.678729Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-10-12T13:59:55.687281Z",
     "iopub.status.busy": "2025-10-12T13:59:55.686911Z",
     "iopub.status.idle": "2025-10-12T13:59:56.229051Z",
     "shell.execute_reply": "2025-10-12T13:59:56.227946Z",
     "shell.execute_reply.started": "2025-10-12T13:59:55.687243Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv(os.path.join(input_dir, 'data/1_初始数据/train/初始数据.csv'))\n",
    "test = pd.read_csv(os.path.join(input_dir, 'data/1_初始数据/test/初始数据.csv'))\n",
    "\n",
    "train.index = train['Id'].values\n",
    "test.index = test['Id'].values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-10-12T13:59:56.230563Z",
     "iopub.status.busy": "2025-10-12T13:59:56.230231Z",
     "iopub.status.idle": "2025-10-12T13:59:56.241377Z",
     "shell.execute_reply": "2025-10-12T13:59:56.240272Z",
     "shell.execute_reply.started": "2025-10-12T13:59:56.230534Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "def data_cleaning(data):\n",
    "    data['dependency']=np.sqrt(data['SQBdependency'])\n",
    "    data['rez_esc']=data['rez_esc'].fillna(0)\n",
    "    data['v18q1']=data['v18q1'].fillna(0)\n",
    "    data['v2a1']=data['v2a1'].fillna(0)\n",
    "    \n",
    "    conditions = [\n",
    "    (data['edjefe']=='no') & (data['edjefa']=='no'), \n",
    "    (data['edjefe']=='yes') & (data['edjefa']=='no'), \n",
    "    (data['edjefe']=='no') & (data['edjefa']=='yes'), \n",
    "    (data['edjefe']!='no') & (data['edjefe']!='yes') & (data['edjefa']=='no'), \n",
    "    (data['edjefe']=='no') & (data['edjefa']!='no') \n",
    "    ]\n",
    "    choices = [0, 1, 1, data['edjefe'], data['edjefa']]\n",
    "    data['edjefx']=np.select(conditions, choices)\n",
    "    data['edjefx']=data['edjefx'].astype(int)\n",
    "    data.drop(['edjefe', 'edjefa'], axis=1, inplace=True)\n",
    "    \n",
    "    meaneduc_nan=data[data['meaneduc'].isnull()][['Id','idhogar','escolari']]\n",
    "    me=meaneduc_nan.groupby('idhogar')['escolari'].mean().reset_index()\n",
    "    for row in meaneduc_nan.iterrows():\n",
    "        idx=row[0]\n",
    "        idhogar=row[1]['idhogar']\n",
    "        m=me[me['idhogar']==idhogar]['escolari'].tolist()[0]\n",
    "        data.at[idx, 'meaneduc']=m\n",
    "        data.at[idx, 'SQBmeaned']=m*m\n",
    "        \n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-10-12T13:59:56.242950Z",
     "iopub.status.busy": "2025-10-12T13:59:56.242615Z",
     "iopub.status.idle": "2025-10-12T13:59:56.370839Z",
     "shell.execute_reply": "2025-10-12T13:59:56.369685Z",
     "shell.execute_reply.started": "2025-10-12T13:59:56.242921Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "train = data_cleaning(train)\n",
    "test = data_cleaning(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-10-12T13:59:56.372436Z",
     "iopub.status.busy": "2025-10-12T13:59:56.372089Z",
     "iopub.status.idle": "2025-10-12T13:59:56.399951Z",
     "shell.execute_reply": "2025-10-12T13:59:56.398682Z",
     "shell.execute_reply.started": "2025-10-12T13:59:56.372407Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "train = train.query('parentesco1==1')\n",
    "train = train.drop('parentesco1', axis=1)\n",
    "test = test.drop('parentesco1', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-10-12T13:59:56.403812Z",
     "iopub.status.busy": "2025-10-12T13:59:56.403449Z",
     "iopub.status.idle": "2025-10-12T13:59:56.410989Z",
     "shell.execute_reply": "2025-10-12T13:59:56.409949Z",
     "shell.execute_reply.started": "2025-10-12T13:59:56.403783Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "def get_numeric(data, status_name):\n",
    "    status_cols = [s for s in data.columns.tolist() if status_name in s]\n",
    "    print('status column names')\n",
    "    print(status_cols)\n",
    "    status_df = data[status_cols]\n",
    "    status_df.columns = list(range(status_df.shape[1]))\n",
    "    status_numeric = status_df.idxmax(1)\n",
    "    status_numeric.name = status_name\n",
    "    data = pd.concat([data, status_numeric], axis=1)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-10-12T13:59:56.412743Z",
     "iopub.status.busy": "2025-10-12T13:59:56.412355Z",
     "iopub.status.idle": "2025-10-12T13:59:56.709233Z",
     "shell.execute_reply": "2025-10-12T13:59:56.708109Z",
     "shell.execute_reply.started": "2025-10-12T13:59:56.412704Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "status column names\n",
      "['epared1', 'epared2', 'epared3']\n",
      "status column names\n",
      "['epared1', 'epared2', 'epared3']\n",
      "status column names\n",
      "['etecho1', 'etecho2', 'etecho3']\n",
      "status column names\n",
      "['etecho1', 'etecho2', 'etecho3']\n",
      "status column names\n",
      "['eviv1', 'eviv2', 'eviv3']\n",
      "status column names\n",
      "['eviv1', 'eviv2', 'eviv3']\n",
      "status column names\n",
      "['instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']\n",
      "status column names\n",
      "['instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']\n"
     ]
    }
   ],
   "source": [
    "status_name_list = ['epared', 'etecho', 'eviv', 'instlevel']\n",
    "for status_name in status_name_list:\n",
    "    train = get_numeric(train, status_name)\n",
    "    test = get_numeric(test, status_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-10-12T13:59:56.711268Z",
     "iopub.status.busy": "2025-10-12T13:59:56.710822Z",
     "iopub.status.idle": "2025-10-12T13:59:56.750444Z",
     "shell.execute_reply": "2025-10-12T13:59:56.749442Z",
     "shell.execute_reply.started": "2025-10-12T13:59:56.711226Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "needless_cols = ['r4t3', 'tamhog', 'hogar_total', 'hhsize', 'v18q', 'sanitario1', 'agesq',\n",
    "                 'mobilephone', 'area1', 'female', 'epared1', 'epared2',\n",
    "                 'epared3', 'etecho1', 'etecho2', 'etecho3',\n",
    "                 'eviv1', 'eviv2', 'eviv3', 'instlevel1', 'instlevel2',\n",
    "                 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6',\n",
    "                 'instlevel7', 'instlevel8', 'instlevel9', 'abastaguafuera']\n",
    "SQB_cols = [s for s in train.columns.tolist() if 'SQB' in s]\n",
    "parentesco_cols = [s for s in train.columns.tolist() if 'parentesco' in s]\n",
    "\n",
    "needless_cols.extend(SQB_cols)\n",
    "needless_cols.extend(parentesco_cols)\n",
    "\n",
    "train = train.drop(needless_cols, axis=1)\n",
    "test = test.drop(needless_cols, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-10-12T13:59:56.752030Z",
     "iopub.status.busy": "2025-10-12T13:59:56.751704Z",
     "iopub.status.idle": "2025-10-12T13:59:56.881580Z",
     "shell.execute_reply": "2025-10-12T13:59:56.880190Z",
     "shell.execute_reply.started": "2025-10-12T13:59:56.752000Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "feature columns \n",
      " 140 -> 94\n"
     ]
    }
   ],
   "source": [
    "ori_train = pd.read_csv(os.path.join(input_dir, 'data/1_初始数据/train/初始数据.csv'))\n",
    "ori_train_X = ori_train.drop(['Id', 'Target', 'idhogar'], axis=1)\n",
    "\n",
    "train_X = train.drop(['Id', 'Target', 'idhogar'], axis=1)\n",
    "\n",
    "print('feature columns \\n {} -> {}'.format(ori_train_X.shape[1], train_X.shape[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-10-12T13:59:56.883418Z",
     "iopub.status.busy": "2025-10-12T13:59:56.883020Z",
     "iopub.status.idle": "2025-10-12T13:59:56.924706Z",
     "shell.execute_reply": "2025-10-12T13:59:56.923781Z",
     "shell.execute_reply.started": "2025-10-12T13:59:56.883385Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "train_Id = train['Id']\n",
    "train_idhogar = train['idhogar']\n",
    "train_y = train['Target']\n",
    "train_X = train.drop(['Id', 'Target', 'idhogar'], axis=1)\n",
    "\n",
    "test_Id = test['Id']\n",
    "test_idhogar = test['idhogar']\n",
    "test_X = test.drop(['Id', 'idhogar'], axis=1)\n",
    "\n",
    "all_Id = pd.concat([train_Id, test_Id], axis=0, sort=False)\n",
    "all_idhogar = pd.concat([train_idhogar, test_idhogar], axis=0, sort=False)\n",
    "all_X = pd.concat([train_X, test_X], axis=0, sort=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-10-12T13:59:58.930941Z",
     "iopub.status.busy": "2025-10-12T13:59:58.930265Z",
     "iopub.status.idle": "2025-10-12T13:59:59.251597Z",
     "shell.execute_reply": "2025-10-12T13:59:59.250807Z",
     "shell.execute_reply.started": "2025-10-12T13:59:58.930898Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "from zhx_config import zhx_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "pickle.dump(train_X , open('./train_X.pkl' , 'wb'))\n",
    "pickle.dump(train_y , open('./train_y.pkl' , 'wb'))\n",
    "pickle.dump(test_X , open('./test_X.pkl' , 'wb'))"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [
    {
     "databundleVersionId": 59638,
     "sourceId": 9840,
     "sourceType": "competition"
    }
   ],
   "dockerImageVersionId": 30357,
   "isGpuEnabled": false,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
